Home Explore Help
Register Sign In
J1X
/
huginn
Watch 2
Star 0
Fork 0

Browse Source
Add LocalFileAgent and introduce concept to handle files

The idea is to have source agents (`LocalFile` and `S3Agent` for now), which just emit a "file_pointer" and consuming
agents (`ReadFile` and `CsvAgent`) which get access to an `IO` object to actually access the file contents. This way we
do not have to pass megabytes of data through the database and the consuming agents can work with relatively unlimited
file sizes.

Dominik Sander 8 years ago
parent
bf7c2feba4
commit
38f1044078
11 changed files with 576 additions and 3 deletions
Show Diff Stats
  1. 3 1
      Gemfile
  2. 1 0
      Gemfile.lock
  3. 58 0
      app/concerns/file_handling.rb
  4. 5 1
      app/concerns/working_helpers.rb
  5. 190 0
      app/models/agents/local_file_agent.rb
  6. 1 0
      lib/agent_runner.rb
  7. 1 1
      lib/utils.rb
  8. 1 0
      spec/env.test
  9. 276 0
      spec/models/agents/local_file_agent_spec.rb
  10. 16 0
      spec/support/shared_examples/file_handling_consumer.rb
  11. 24 0
      spec/support/shared_examples/working_helpers.rb

+ 3 - 1
Gemfile
View File 

@@ -63,6 +63,9 @@ gem 'haversine'
63 63 
 gem 'omniauth-evernote'
64 64 
 gem 'evernote_oauth'
65 65
66 
+# LocalFileAgent (watch functionality)
67 
+gem 'listen', '~> 3.0.5', require: false
68 
+
66 69 
 # Optional Services.
67 70 
 gem 'omniauth-37signals'          # BasecampAgent
68 71 
 gem 'omniauth-wunderlist', github: 'wunderlist/omniauth-wunderlist', ref: 'd0910d0396107b9302aa1bc50e74bb140990ccb8'
 
@@ -75,7 +78,6 @@ unless Gem::Version.new(Bundler::VERSION) >= Gem::Version.new('1.5.0')
75 78 
 end
76 79
77 80 
 gem 'protected_attributes', '~>1.0.8' # This must be loaded before some other gems, like delayed_job.
78 
-
79 81 
 gem 'ace-rails-ap', '~> 2.0.1'
80 82 
 gem 'bootstrap-kaminari-views', '~> 0.0.3'
81 83 
 gem 'bundler', '>= 1.5.0'

+ 1 - 0
Gemfile.lock
View File 

@@ -621,6 +621,7 @@ DEPENDENCIES
621 621 
   kramdown (~> 1.3.3)
622 622 
   letter_opener_web
623 623 
   liquid (~> 3.0.3)
624 
+  listen (~> 3.0.5)
624 625 
   mini_magick
625 626 
   mqtt
626 627 
   multi_xml

+ 58 - 0
app/concerns/file_handling.rb
View File 

@@ -0,0 +1,58 @@
1 
+module FileHandling
2 
+  extend ActiveSupport::Concern
3 
+
4 
+  def get_file_pointer(file)
5 
+    { file_pointer: { file: file, agent_id: id } }
6 
+  end
7 
+
8 
+  def get_io(event)
9 
+    return nil unless event.payload['file_pointer'] &&
10 
+                      event.payload['file_pointer']['file'] &&
11 
+                      event.payload['file_pointer']['agent_id']
12 
+    event.user.agents.find(event.payload['file_pointer']['agent_id']).get_io(event.payload['file_pointer']['file'])
13 
+  end
14 
+
15 
+  def emitting_file_handling_agent_description
16 
+    @emitting_file_handling_agent_description ||=
17 
+      "This agent only emits a 'file pointer', not the data inside the files, the following agents can consume the created events: `#{receiving_file_handling_agents.join('`, `')}`. Read more about the concept in the [wiki](https://github.com/cantino/huginn/wiki/How-Huginn-works-with-files)."
18 
+  end
19 
+
20 
+  def receiving_file_handling_agent_description
21 
+    @receiving_file_handling_agent_description ||=
22 
+      "This agent can consume a 'file pointer' event from the following agents with no additional configuration: `#{emitting_file_handling_agents.join('`, `')}`. Read more about the concept in the [wiki](https://github.com/cantino/huginn/wiki/How-Huginn-works-with-files)."
23 
+  end
24 
+
25 
+  private
26 
+
27 
+  def emitting_file_handling_agents
28 
+    emitting_file_handling_agents = file_handling_agents.select { |a| a.emits_file_pointer? }
29 
+    emitting_file_handling_agents.map { |a| a.to_s.demodulize }
30 
+  end
31 
+
32 
+  def receiving_file_handling_agents
33 
+    receiving_file_handling_agents = file_handling_agents.select { |a| a.consumes_file_pointer? }
34 
+    receiving_file_handling_agents.map { |a| a.to_s.demodulize }
35 
+  end
36 
+
37 
+  def file_handling_agents
38 
+    @file_handling_agents ||= Agent.types.select{ |c| c.included_modules.include?(FileHandling) }.map { |d| d.name.constantize }
39 
+  end
40 
+
41 
+  module ClassMethods
42 
+    def emits_file_pointer!
43 
+      @emits_file_pointer = true
44 
+    end
45 
+
46 
+    def emits_file_pointer?
47 
+      !!@emits_file_pointer
48 
+    end
49 
+
50 
+    def consumes_file_pointer!
51 
+      @consumes_file_pointer = true
52 
+    end
53 
+
54 
+    def consumes_file_pointer?
55 
+      !!@consumes_file_pointer
56 
+    end
57 
+  end
58 
+end

+ 5 - 1
app/concerns/working_helpers.rb
View File 

@@ -12,4 +12,8 @@ module WorkingHelpers
12 12 
   def received_event_without_error?
13 13 
     (last_receive_at.present? && last_error_log_at.blank?) || (last_receive_at.present? && last_error_log_at.present? && last_receive_at > last_error_log_at)
14 14 
   end
15 
-end
15 
+
16 
+  def checked_without_error?
17 
+    (last_check_at.present? && last_error_log_at.nil?) || (last_check_at.present? && last_error_log_at.present? && last_check_at > last_error_log_at)
18 
+  end
19 
+end

+ 190 - 0
app/models/agents/local_file_agent.rb
View File 

@@ -0,0 +1,190 @@
1 
+module Agents
2 
+  class LocalFileAgent < Agent
3 
+    include LongRunnable
4 
+    include FormConfigurable
5 
+    include FileHandling
6 
+
7 
+    emits_file_pointer!
8 
+
9 
+    default_schedule 'every_1h'
10 
+
11 
+    def self.should_run?
12 
+      ENV['ENABLE_INSECURE_AGENTS'] == "true"
13 
+    end
14 
+
15 
+    description do
16 
+      <<-MD
17 
+        The LocalFileAgent can watch a file/directory for changes or emit an event for every file in that directory. When receiving an event it writes the received data into a file.
18 
+
19 
+        `mode` determines if the agent is emitting events for (changed) files or writing received event data to disk.
20 
+
21 
+        ### Reading
22 
+
23 
+        When `watch` is set to `true` the LocalFileAgent will watch the specified `path` for changes, the schedule is ignored and the file system is watched continuously. An event will be emitted for every detected change.
24 
+
25 
+        When `watch` is set to `false` the agent will emit an event for every file in the directory on each scheduled run.
26 
+
27 
+        #{emitting_file_handling_agent_description}
28 
+
29 
+        ### Writing
30 
+
31 
+        Every event will be writting into a file at `path`, Liquid interpolation is possible to change the path per event.
32 
+
33 
+        When `append` is true the received data will be appended to the file.
34 
+
35 
+        Use [Liquid](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid) templating in `data` to specify which part of the received event should be written.
36 
+
37 
+        *Warning*: This type of Agent can read and write any file the user that runs the Huginn server has access to, and is #{Agents::LocalFileAgent.should_run? ? "**currently enabled**" : "**currently disabled**"}.
38 
+        Only enable this Agent if you trust everyone using your Huginn installation.
39 
+        You can enable this Agent in your .env file by setting `ENABLE_INSECURE_AGENTS` to `true`.
40 
+      MD
41 
+    end
42 
+
43 
+    event_description do
44 
+      "Events will looks like this:\n\n    %s" % if boolify(interpolated['watch'])
45 
+        Utils.pretty_print(
46 
+          "file_pointer" => {
47 
+            "file" => "/tmp/test/filename",
48 
+            "agent_id" => id
49 
+          },
50 
+          "event_type" => "modified/added/removed"
51 
+        )
52 
+      else
53 
+        Utils.pretty_print(
54 
+          "file_pointer" => {
55 
+            "file" => "/tmp/test/filename",
56 
+            "agent_id" => id
57 
+          }
58 
+        )
59 
+      end
60 
+    end
61 
+
62 
+    def default_options
63 
+      {
64 
+        'mode' => 'read',
65 
+        'watch' => 'true',
66 
+        'append' => 'false',
67 
+        'path' => "",
68 
+        'data' => '{{ data }}'
69 
+      }
70 
+    end
71 
+
72 
+    form_configurable :mode, type: :array, values: %w(read write)
73 
+    form_configurable :watch, type: :array, values: %w(true false)
74 
+    form_configurable :path, type: :string
75 
+    form_configurable :append, type: :boolean
76 
+    form_configurable :data, type: :string
77 
+
78 
+    def validate_options
79 
+      if options['mode'].blank? || !['read', 'write'].include?(options['mode'])
80 
+        errors.add(:base, "The 'mode' option is required and must be set to 'read' or 'write'")
81 
+      end
82 
+      if options['watch'].blank? || ![true, false].include?(boolify(options['watch']))
83 
+        errors.add(:base, "The 'watch' option is required and must be set to 'true' or 'false'")
84 
+      end
85 
+      if options['append'].blank? || ![true, false].include?(boolify(options['append']))
86 
+        errors.add(:base, "The 'append' option is required and must be set to 'true' or 'false'")
87 
+      end
88 
+      if options['path'].blank?
89 
+        errors.add(:base, "The 'path' option is required.")
90 
+      end
91 
+    end
92 
+
93 
+    def working?
94 
+      should_run?(false) && ((interpolated['mode'] == 'read' && check_path_existance && checked_without_error?) ||
95 
+                             (interpolated['mode'] == 'write' && received_event_without_error?))
96 
+    end
97 
+
98 
+    def check
99 
+      return if interpolated['mode'] != 'read' || boolify(interpolated['watch']) || !should_run?
100 
+      return unless check_path_existance(true)
101 
+      if File.directory?(expanded_path)
102 
+        Dir.glob(File.join(expanded_path, '*')).select { |f| File.file?(f) }
103 
+      else
104 
+        [expanded_path]
105 
+      end.each do |file|
106 
+        create_event payload: get_file_pointer(file)
107 
+      end
108 
+    end
109 
+
110 
+    def receive(incoming_events)
111 
+      return if interpolated['mode'] != 'write' || !should_run?
112 
+      incoming_events.each do |event|
113 
+        mo = interpolated(event)
114 
+        File.open(File.expand_path(mo['path']), boolify(mo['append']) ? 'a' : 'w') do |file|
115 
+          file.write(mo['data'])
116 
+        end
117 
+      end
118 
+    end
119 
+
120 
+    def start_worker?
121 
+      interpolated['mode'] == 'read' && boolify(interpolated['watch']) && should_run? && check_path_existance
122 
+    end
123 
+
124 
+    def check_path_existance(log = true)
125 
+      if !File.exist?(expanded_path)
126 
+        error("File or directory '#{expanded_path}' does not exist") if log
127 
+        return false
128 
+      end
129 
+      true
130 
+    end
131 
+
132 
+    def get_io(file)
133 
+      File.open(file, 'r')
134 
+    end
135 
+
136 
+    def expanded_path
137 
+      @expanded_path ||= File.expand_path(interpolated['path'])
138 
+    end
139 
+
140 
+    private
141 
+
142 
+    def should_run?(log = true)
143 
+      if self.class.should_run?
144 
+        true
145 
+      else
146 
+        error("Unable to run because insecure agents are not enabled. Set ENABLE_INSECURE_AGENTS to true in the Huginn .env configuration.") if log
147 
+        false
148 
+      end
149 
+    end
150 
+
151 
+    class Worker < LongRunnable::Worker
152 
+      def setup
153 
+        require 'listen'
154 
+        @listener = Listen.to(*listen_options, &method(:callback))
155 
+      end
156 
+
157 
+      def run
158 
+        sleep unless agent.check_path_existance(true)
159 
+
160 
+        @listener.start
161 
+        sleep
162 
+      end
163 
+
164 
+      def stop
165 
+        @listener.stop
166 
+      end
167 
+
168 
+      private
169 
+
170 
+      def callback(*changes)
171 
+        AgentRunner.with_connection do
172 
+          changes.zip([:modified, :added, :removed]).each do |files, event_type|
173 
+            files.each do |file|
174 
+              agent.create_event payload: agent.get_file_pointer(file).merge(event_type: event_type)
175 
+            end
176 
+          end
177 
+          agent.touch(:last_check_at)
178 
+        end
179 
+      end
180 
+
181 
+      def listen_options
182 
+        if File.directory?(agent.expanded_path)
183 
+          [agent.expanded_path, ignore!: [] ]
184 
+        else
185 
+          [File.dirname(agent.expanded_path), { ignore!: [], only: /\A#{Regexp.escape(File.basename(agent.expanded_path))}\z/ } ]
186 
+        end
187 
+      end
188 
+    end
189 
+  end
190 
+end

+ 1 - 0
lib/agent_runner.rb
View File 

@@ -118,5 +118,6 @@ end
118 118
119 119 
 require 'agents/twitter_stream_agent'
120 120 
 require 'agents/jabber_agent'
121 
+require 'agents/local_file_agent'
121 122 
 require 'huginn_scheduler'
122 123 
 require 'delayed_job_worker'

+ 1 - 1
lib/utils.rb
View File 

@@ -15,7 +15,7 @@ module Utils
15 15 
   def self.pretty_print(struct, indent = true)
16 16 
     output = JSON.pretty_generate(struct)
17 17 
     if indent
18 
-      output.gsub(/\n/i, "\n    ").tap { |a| p a }
18 
+      output.gsub(/\n/i, "\n    ")
19 19 
     else
20 20 
       output
21 21 
     end

+ 1 - 0
spec/env.test
View File 

@@ -12,3 +12,4 @@ EVERNOTE_OAUTH_KEY=evernoteoauthkey
12 12 
 EVERNOTE_OAUTH_SECRET=evernoteoauthsecret
13 13 
 FAILED_JOBS_TO_KEEP=2
14 14 
 REQUIRE_CONFIRMED_EMAIL=false
15 
+ENABLE_INSECURE_AGENTS=true

+ 276 - 0
spec/models/agents/local_file_agent_spec.rb
View File 

@@ -0,0 +1,276 @@
1 
+require 'rails_helper'
2 
+
3 
+describe Agents::LocalFileAgent do
4 
+  before(:each) do
5 
+    @valid_params = {
6 
+                      'mode' => 'read',
7 
+                      'watch' => 'false',
8 
+                      'append' => 'false',
9 
+                      'path' => File.join(Rails.root, 'tmp', 'spec')
10 
+                    }
11 
+    FileUtils.mkdir_p File.join(Rails.root, 'tmp', 'spec')
12 
+
13 
+    @checker = Agents::LocalFileAgent.new(:name => "somename", :options => @valid_params)
14 
+    @checker.user = users(:jane)
15 
+    @checker.save!
16 
+  end
17 
+
18 
+  after(:all) do
19 
+    FileUtils.rm_r File.join(Rails.root, 'tmp', 'spec')
20 
+  end
21 
+
22 
+  describe "#validate_options" do
23 
+    it "is valid with the given options" do
24 
+      expect(@checker).to be_valid
25 
+    end
26 
+
27 
+    it "requires mode to be either 'read' or 'write'" do
28 
+      @checker.options['mode'] = 'write'
29 
+      expect(@checker).to be_valid
30 
+      @checker.options['mode'] = 'write'
31 
+      expect(@checker).to be_valid
32 
+      @checker.options['mode'] = 'test'
33 
+      expect(@checker).not_to be_valid
34 
+    end
35 
+
36 
+    it "requires the path to be set" do
37 
+      @checker.options['path'] = ''
38 
+      expect(@checker).not_to be_valid
39 
+    end
40 
+
41 
+    it "requires watch to be present" do
42 
+      @checker.options['watch'] = ''
43 
+      expect(@checker).not_to be_valid
44 
+    end
45 
+
46 
+    it "requires watch to be either 'true' or 'false'" do
47 
+      @checker.options['watch'] = 'true'
48 
+      expect(@checker).to be_valid
49 
+      @checker.options['watch'] = 'false'
50 
+      expect(@checker).to be_valid
51 
+      @checker.options['watch'] = 'test'
52 
+      expect(@checker).not_to be_valid
53 
+    end
54 
+
55 
+    it "requires append to be either 'true' or 'false'" do
56 
+      @checker.options['append'] = 'true'
57 
+      expect(@checker).to be_valid
58 
+      @checker.options['append'] = 'false'
59 
+      expect(@checker).to be_valid
60 
+      @checker.options['append'] = 'test'
61 
+      expect(@checker).not_to be_valid
62 
+    end
63 
+  end
64 
+
65 
+  context "#working" do
66 
+    it "is working with no recent errors in read mode" do
67 
+      @checker.last_check_at = Time.now
68 
+      expect(@checker).to be_working
69 
+    end
70 
+
71 
+    it "is working with no recent errors in write mode" do
72 
+      @checker.options['mode'] = 'write'
73 
+      @checker.last_receive_at = Time.now
74 
+      expect(@checker).to be_working
75 
+    end
76 
+  end
77 
+
78 
+  context "#check_path_existance" do
79 
+    it "is truethy when the path exists" do
80 
+      expect(@checker.check_path_existance).to be_truthy
81 
+    end
82 
+
83 
+    it "is falsy when the path does not exist" do
84 
+      @checker.options['path'] = '/doesnotexist'
85 
+      expect(@checker.check_path_existance).to be_falsy
86 
+    end
87 
+
88 
+    it "create a log entry" do
89 
+      @checker.options['path'] = '/doesnotexist'
90 
+      expect { @checker.check_path_existance(true) }.to change(AgentLog, :count).by(1)
91 
+    end
92 
+
93 
+    it "works with non-expanded paths" do
94 
+      @checker.options['path'] = '~'
95 
+      expect(@checker.check_path_existance).to be_truthy
96 
+    end
97 
+  end
98 
+
99 
+  def with_files(*files)
100 
+    files.each { |f| FileUtils.touch(f) }
101 
+    yield
102 
+    files.each { |f| FileUtils.rm(f) }
103 
+  end
104 
+
105 
+  context "#check" do
106 
+    it "does not create events when the directory is empty" do
107 
+      expect { @checker.check }.to change(Event, :count).by(0)
108 
+    end
109 
+
110 
+    it "creates an event for every file in the directory" do
111 
+      with_files(File.join(Rails.root, 'tmp', 'spec', 'one'), File.join(Rails.root, 'tmp', 'spec', 'two')) do
112 
+        expect { @checker.check }.to change(Event, :count).by(2)
113 
+        expect(Event.last.payload.has_key?('file_pointer')).to be_truthy
114 
+      end
115 
+    end
116 
+
117 
+    it "creates an event if the configured file exists" do
118 
+      @checker.options['path'] = File.join(Rails.root, 'tmp', 'spec', 'one')
119 
+      with_files(File.join(Rails.root, 'tmp', 'spec', 'one'), File.join(Rails.root, 'tmp', 'spec', 'two')) do
120 
+        expect { @checker.check }.to change(Event, :count).by(1)
121 
+        payload = Event.last.payload
122 
+        expect(payload.has_key?('file_pointer')).to be_truthy
123 
+        expect(payload['file_pointer']['file']).to eq(@checker.options['path'])
124 
+      end
125 
+    end
126 
+
127 
+    it "does not run when ENABLE_INSECURE_AGENTS is not set to true" do
128 
+      ENV['ENABLE_INSECURE_AGENTS'] = 'false'
129 
+      expect { @checker.check }.to change(AgentLog, :count).by(1)
130 
+      ENV['ENABLE_INSECURE_AGENTS'] = 'true'
131 
+    end
132 
+  end
133 
+
134 
+  context "#event_description" do
135 
+    it "should include event_type when watch is set to true" do
136 
+      @checker.options['watch'] = 'true'
137 
+      expect(@checker.event_description).to include('event_type')
138 
+    end
139 
+
140 
+    it "should not include event_type when watch is set to false" do
141 
+      @checker.options['watch'] = 'false'
142 
+      expect(@checker.event_description).not_to include('event_type')
143 
+    end
144 
+  end
145 
+
146 
+  it "get_io opens the file" do
147 
+    mock(File).open('test', 'r')
148 
+    @checker.get_io('test')
149 
+  end
150 
+
151 
+  context "#start_worker?" do
152 
+    it "reeturns true when watch is true" do
153 
+      @checker.options['watch'] = 'true'
154 
+      expect(@checker.start_worker?).to be_truthy
155 
+    end
156 
+
157 
+    it "returns false when watch is false" do
158 
+      @checker.options['watch'] = 'false'
159 
+      expect(@checker.start_worker?).to be_falsy
160 
+    end
161 
+  end
162 
+
163 
+  context "#receive" do
164 
+    before(:each) do
165 
+      @checker.options['mode'] = 'write'
166 
+      @checker.options['data'] = '{{ data }}'
167 
+      @file_mock = mock()
168 
+    end
169 
+
170 
+    it "writes the data at data into a file" do
171 
+      mock(@file_mock).write('hello world')
172 
+      event = Event.new(payload: {'data' => 'hello world'})
173 
+      mock(File).open(File.join(Rails.root, 'tmp', 'spec'), 'w').yields @file_mock
174 
+      @checker.receive([event])
175 
+    end
176 
+
177 
+    it "appends the data at data onto a file" do
178 
+      mock(@file_mock).write('hello world')
179 
+      @checker.options['append'] = 'true'
180 
+      event = Event.new(payload: {'data' => 'hello world'})
181 
+      mock(File).open(File.join(Rails.root, 'tmp', 'spec'), 'a').yields @file_mock
182 
+      @checker.receive([event])
183 
+    end
184 
+
185 
+    it "does not receive when ENABLE_INSECURE_AGENTS is not set to true" do
186 
+      ENV['ENABLE_INSECURE_AGENTS'] = 'false'
187 
+      expect { @checker.receive([]) }.to change(AgentLog, :count).by(1)
188 
+      ENV['ENABLE_INSECURE_AGENTS'] = 'true'
189 
+    end
190 
+  end
191 
+
192 
+  describe describe Agents::LocalFileAgent::Worker do
193 
+    require 'listen'
194 
+
195 
+    before(:each) do
196 
+      @checker.options['watch'] = true
197 
+      @checker.save
198 
+      @worker = Agents::LocalFileAgent::Worker.new(agent: @checker)
199 
+      @listen_mock = mock()
200 
+    end
201 
+
202 
+    context "#setup" do
203 
+      it "initializes the listen gem" do
204 
+        mock(Listen).to(@checker.options['path'], ignore!: [])
205 
+        @worker.setup
206 
+      end
207 
+    end
208 
+
209 
+    context "#run" do
210 
+      before(:each) do
211 
+        stub(Listen).to { @listen_mock }
212 
+        @worker.setup
213 
+      end
214 
+
215 
+      it "starts to listen to changes in the directory when the path is present" do
216 
+        mock(@worker).sleep
217 
+        mock(@listen_mock).start
218 
+        @worker.run
219 
+      end
220 
+
221 
+      it "does nothing when the path does not exist" do
222 
+        mock(@worker.agent).check_path_existance(true) { false }
223 
+        dont_allow(@listen_mock).start
224 
+        mock(@worker).sleep { raise "Sleeping" }
225 
+        expect { @worker.run }.to raise_exception(RuntimeError, 'Sleeping')
226 
+      end
227 
+    end
228 
+
229 
+    context "#stop" do
230 
+      it "stops the listen gem" do
231 
+        stub(Listen).to { @listen_mock }
232 
+        @worker.setup
233 
+        mock(@listen_mock).stop
234 
+        @worker.stop
235 
+      end
236 
+    end
237 
+
238 
+    context "#callback" do
239 
+      let(:file) { File.join(Rails.root, 'tmp', 'one') }
240 
+      let(:file2) { File.join(Rails.root, 'tmp', 'one2') }
241 
+
242 
+      it "creates an event for modifies files" do
243 
+        expect { @worker.send(:callback, [file], [], [])}.to change(Event, :count).by(1)
244 
+        payload = Event.last.payload
245 
+        expect(payload['event_type']).to eq('modified')
246 
+      end
247 
+
248 
+      it "creates an event for modifies files" do
249 
+        expect { @worker.send(:callback, [], [file], [])}.to change(Event, :count).by(1)
250 
+        payload = Event.last.payload
251 
+        expect(payload['event_type']).to eq('added')
252 
+      end
253 
+
254 
+      it "creates an event for modifies files" do
255 
+        expect { @worker.send(:callback, [], [], [file])}.to change(Event, :count).by(1)
256 
+        payload = Event.last.payload
257 
+        expect(payload['event_type']).to eq('removed')
258 
+      end
259 
+
260 
+      it "creates an event each changed file" do
261 
+        expect { @worker.send(:callback, [], [file], [file2])}.to change(Event, :count).by(2)
262 
+      end
263 
+    end
264 
+
265 
+    context "#listen_options" do
266 
+      it "returns the path when a directory is given" do
267 
+        expect(@worker.send(:listen_options)).to eq([File.join(Rails.root, 'tmp', 'spec'), ignore!: []])
268 
+      end
269 
+
270 
+      it "restricts to only the specified filename" do
271 
+        @worker.agent.options['path'] = File.join(Rails.root, 'tmp', 'one')
272 
+        expect(@worker.send(:listen_options)).to eq([File.join(Rails.root, 'tmp'), { only: /\Aone\z/, ignore!: [] } ])
273 
+      end
274 
+    end
275 
+  end
276 
+end

+ 16 - 0
spec/support/shared_examples/file_handling_consumer.rb
View File 

@@ -0,0 +1,16 @@
1 
+require 'rails_helper'
2 
+
3 
+shared_examples_for 'FileHandlingConsumer' do
4 
+  it 'returns a file pointer' do
5 
+    expect(@checker.get_file_pointer('testfile')).to eq(file_pointer: { file: "testfile", agent_id: @checker.id})
6 
+  end
7 
+
8 
+  it 'get_io raises an exception when trying to access an agent of a different user' do
9 
+    @checker2 = @checker.dup
10 
+    @checker2.user = users(:bob)
11 
+    @checker2.save!
12 
+    expect(@checker2.user.id).not_to eq(@checker.user.id)
13 
+    event = Event.new(user: @checker.user, payload: {'file_pointer' => {'file' => 'test', 'agent_id' => @checker2.id}})
14 
+    expect { @checker.get_io(event) }.to raise_error(ActiveRecord::RecordNotFound)
15 
+  end
16 
+end

+ 24 - 0
spec/support/shared_examples/working_helpers.rb
View File 

@@ -50,4 +50,28 @@ shared_examples_for WorkingHelpers do
50 50 
       expect(@agent.received_event_without_error?).to eq(true)
51 51 
     end
52 52 
   end
53 
+
54 
+  describe "checked_without_error?" do
55 
+    before do
56 
+      @agent = described_class.new
57 
+    end
58 
+
59 
+    it "should return false until the first time check ran" do
60 
+      expect(@agent.checked_without_error?).to eq(false)
61 
+      @agent.last_check_at = Time.now
62 
+      expect(@agent.checked_without_error?).to eq(true)
63 
+    end
64 
+
65 
+    it "should return false when the last error occured after the check" do
66 
+      @agent.last_check_at = Time.now - 1.minute
67 
+      @agent.last_error_log_at = Time.now
68 
+      expect(@agent.checked_without_error?).to eq(false)
69 
+    end
70 
+
71 
+    it "should return true when the last check occured after the last error" do
72 
+      @agent.last_check_at = Time.now
73 
+      @agent.last_error_log_at = Time.now - 1.minute
74 
+      expect(@agent.checked_without_error?).to eq(true)
75 
+    end
76 
+  end
53 77 
 end